<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2016-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Getting Started</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="CDAP Pipelines" href="index.html" />
    <link rel="next" title="CDAP Studio" href="studio.html" />
    <link rel="prev" title="Introduction to CDAP Pipelines" href="concepts-design.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: developer-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../getting-started/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/sandbox/index.html">CDAP Sandbox</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html">二进制 Zip 文件</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/zip.html#cdap-sandbox">启动和停止 CDAP Sandbox</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/virtual-machine.html">虚拟机镜像</a></li>
<li class="toctree-l3"><a class="reference internal" href="../getting-started/sandbox/docker.html">Docker 镜像</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/quick-start.html">快速入门</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/dev-env.html">搭建开发环境</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/start-stop-cdap.html">启动和停止 CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="../getting-started/building-apps.html">构建并运行应用</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../overview/index.html"> 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../overview/anatomy.html"> 大数据应用剖析</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/modes.html"> 模式和组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/abstractions.html"> 核心概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../overview/interfaces.html"> 编程接口</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../building-blocks/index.html"> 抽象概念</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/core.html"> Core Abstractions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/applications.html"> Applications</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/datasets/index.html"> Datasets</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/overview.html"> Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/table.html"> Table API</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/fileset.html"> FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/partitioned-fileset.html"> Partitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/time-partitioned-fileset.html"> TimePartitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/system-custom.html"> System and Custom Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/permissions.html"> Dataset Permissions</a></li>
<li class="toctree-l3"><a class="reference internal" href="../building-blocks/datasets/cube.html"> Cube Dataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/mapreduce-programs.html"> MapReduce Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/plugins.html"> Plugins</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/schedules.html"> Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/secure-keys.html"> Secure Keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/services.html"> Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/spark-programs.html"> Spark Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workers.html"> Workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/workflows.html"> Workflows</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/artifacts.html"> Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/program-lifecycle.html"> Program Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/namespaces.html"> Namespaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transaction-system.html"> Transaction System</a></li>
<li class="toctree-l2"><a class="reference internal" href="../building-blocks/transactional-messaging-system.html"> Transactional Messaging System</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../metadata/index.html"> 元数据</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../metadata/system-metadata.html"> System Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/discovery-lineage.html"> Discovery and Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/field-lineage.html"> Field Level Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/audit-logging.html"> Audit Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/metadata-ui.html"> CDAP Metadata UI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../metadata/programmatic-metadata.html"> Accessing metadata programmatically</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 数据流管道</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="concepts-design.html"> Concepts and Design</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> Getting Started</a></li>
<li class="toctree-l2"><a class="reference internal" href="studio.html"> CDAP Studio</a></li>
<li class="toctree-l2"><a class="reference internal" href="creating-pipelines.html"> Creating Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="running-pipelines.html"> Running Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugin-management.html"> Plugin Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugins/index.html"> Plugin Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="developing-pipelines.html"> Developing Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="developing-plugins/index.html"> Developing Plugins</a><ul>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/plugin-basics.html">Plugin Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/creating-a-plugin.html">Creating a Plugin</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/presentation-plugins.html">Plugin Presentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/testing-plugins.html">Testing Plugins</a></li>
<li class="toctree-l3"><a class="reference internal" href="developing-plugins/packaging-plugins.html">Packaging Plugins</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="how-cdap-pipelines-work.html"> How CDAP Pipelines Work</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../cloud-runtimes/index.html"> 云平台运行</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/concepts/index.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/provisioners/index.html"> Provisioners</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/gcp-dataproc.html">Google Dataproc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/aws-emr.html">Amazon Elastic MapReduce</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/provisioners/remote-hadoop.html">Remote Hadoop</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/profiles/index.html"> Profiles</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/creating-profiles.html">Creating Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/assigning-profiles.html">Assigning Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../cloud-runtimes/profiles/admin-controls.html">Admin Controls</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../cloud-runtimes/example/index.html"> Example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../security/index.html"> 安全</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../security/client-authentication.html">Client Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-java.html">CDAP Authentication Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/cdap-authentication-clients-python.html">CDAP Authentication Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/custom-authentication.html">Custom Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../security/authorization-extensions.html">Authorization Extensions</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../testing/index.html"> 测试和调试</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../testing/testing.html"> Testing a CDAP Application</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/debugging.html"> Debugging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../testing/troubleshooting.html"> Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../ingesting-tools/index.html"> 数据融合</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-java.html">CDAP Stream Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-python.html">CDAP Stream Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-stream-clients-ruby.html">CDAP Stream Client for Ruby</a></li>
<li class="toctree-l2"><a class="reference internal" href="../ingesting-tools/cdap-flume.html">CDAP Flume</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-exploration/index.html"> 数据探索</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/filesets.html"> Fileset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/tables.html"> Table Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/object-mapped-tables.html"> ObjectMappedTable Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/custom-datasets.html"> Custom Dataset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-exploration/hive-execution-engines.html"> Hive Execution Engines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/index.html"> 高级主题</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../advanced/application-logback.html"> Application Logback</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/best-practices.html"> Best Practices</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/class-loading.html"> Class Loading</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/configuring-resources.html"> Configuring Program Resources</a></li>
<li class="toctree-l2"><a class="reference internal" href="../advanced/program-retry-policies.html"> Program Retry Policies</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="getting-started">
<span id="cdap-pipelines-getting-started"></span><h1>Getting Started<a class="headerlink" href="#getting-started" title="Permalink to this headline">🔗</a></h1>
<p>This is a quick tutorial, covering the basics of CDAP pipelines:</p>
<blockquote>
<div><ol class="arabic simple">
<li><a class="reference internal" href="#install-cdap">Install CDAP</a></li>
<li><a class="reference internal" href="#creating-cdap-pipelines">Creating CDAP Pipelines</a></li>
<li><a class="reference internal" href="#start-cdap-studio">Start CDAP Studio</a></li>
<li><a class="reference internal" href="#quick-tour-of-cdap-studio">Quick Tour of CDAP Studio</a></li>
<li><a class="reference internal" href="#creating-a-simple-pipeline">Creating a Simple Pipeline</a></li>
</ol>
</div></blockquote>
<div class="section" id="install-cdap">
<span id="cdap-pipelines-getting-started-install"></span><h2>Install CDAP<a class="headerlink" href="#install-cdap" title="Permalink to this headline">🔗</a></h2>
<ul class="simple">
<li>If you haven’t already, <a class="reference internal" href="../getting-started/sandbox/index.html#sandbox-index"><span class="std std-ref">download and install the CDAP Sandbox</span></a>.</li>
<li><a class="reference internal" href="../getting-started/start-stop-cdap.html#start-stop-cdap"><span class="std std-ref">Start CDAP</span></a>, open up a web browser, and go to the <a class="reference external" href="http://localhost:11011/oldcdap/ns/default/">CDAP UI</a>.</li>
</ul>
</div>
<div class="section" id="creating-cdap-pipelines">
<span id="cdap-pipelines-getting-started-start"></span><h2>Creating CDAP Pipelines<a class="headerlink" href="#creating-cdap-pipelines" title="Permalink to this headline">🔗</a></h2>
<ul>
<li><p class="first">To start creating a CDAP pipeline, either select <em>Pipelines</em> from the pull-down menu in the upper
left, or go to the <a class="reference external" href="http://localhost:11011/oldcdap/ns/default/pipelines/">CDAP Pipelines URL</a>.</p>
<p>This will take you to a list of pipelines, which will most likely be empty:</p>
<div class="figure align-center" id="id1" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-no-pipelines.png"><img alt="../_images/cdap-pipelines-no-pipelines.png" class="bordered-image" src="../_images/cdap-pipelines-no-pipelines.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> List of pipelines</span></p>
</div>
</li>
</ul>
</div>
<div class="section" id="start-cdap-studio">
<span id="cdap-pipelines-getting-started-cdap-studio"></span><h2>Start CDAP Studio<a class="headerlink" href="#start-cdap-studio" title="Permalink to this headline">🔗</a></h2>
<ul>
<li><p class="first">To start <em>CDAP Studio</em> and start creating a pipeline, do any of these:</p>
<ul class="simple">
<li>Click the <em>+</em> button to add a new pipeline;</li>
<li>Click the tool bar label <em>Studio</em> (to the left of the label <em>Pipelines</em>); or</li>
<li>Go to the <a class="reference external" href="http://localhost:11011/oldcdap/ns/default/pipelines/studio?artifactType=cdap-data-pipeline">CDAP Studio URL</a></li>
</ul>
<p>Studio will start and you will be creating a new pipeline, by default the first
pipeline type in the menu, a <em>Data Pipeline - Batch</em>, which is a batch-type pipeline:</p>
<div class="figure align-center" id="id2" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-studio-empty.png"><img alt="../_images/cdap-studio-empty.png" class="bordered-image" src="../_images/cdap-studio-empty.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Empty canvas for creating pipelines</span></p>
</div>
</li>
</ul>
</div>
<div class="section" id="quick-tour-of-cdap-studio">
<span id="cdap-pipelines-getting-started-studio-tour"></span><h2>Quick Tour of CDAP Studio<a class="headerlink" href="#quick-tour-of-cdap-studio" title="Permalink to this headline">🔗</a></h2>
<p>Before we begin an example, let’s look at Studio and its different components:</p>
<div class="figure align-center" id="id3" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-studio-annotated.png"><img alt="../_images/cdap-studio-annotated.png" class="bordered-image" src="../_images/cdap-studio-annotated.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Annotations showing components</span></p>
</div>
<p>In the upper-left is a menu that specifies which <strong>application template artifact</strong> you are
currently using for your pipeline. For now, we’ll leave it with the default, <em>Data
Pipeline - Batch</em>.</p>
<p>In the left sidebar are icons representing the different <strong>available plugins</strong> that work
with the current application template. They are grouped into different categories, and are
revealed by clicking the disclosure triangles to the left of each category label
(<em>Source, Transform, Analytics, Sink, Action</em>).</p>
<p>In the middle is the grey, gridded <strong>studio canvas</strong>, used to create the pipeline on by
clicking an available plugin in the left sidebar to add the plugin’s icon to the canvas, and
then by dragging the icon into position.</p>
<p>The image shows an existing <strong>pipeline</strong>, with three <strong>plugin icons</strong> in place and the
first two connected.</p>
<p>Note that icons are of different <strong>colors</strong> and <strong>shapes</strong> with differently shaped <strong>connection
nodes</strong> (either <strong>circular</strong> for data connections or <strong>square</strong> for control connections):</p>
<ul class="simple">
<li><strong>Green:</strong> a data <strong>generator</strong>, with a single <strong>right-side</strong> data connection node, such
as a <em>source</em> plugin</li>
<li><strong>Blue:</strong> a data <strong>receiver and generator</strong>, with <strong>both left- and right-side</strong> data
connection nodes, such as a <em>transform</em> or <em>analytic</em> plugin</li>
<li><strong>Purple:</strong> a data <strong>receiver</strong>, with only a <strong>left-side</strong> data connection node, such as
a <em>sink</em> plugin</li>
<li><strong>Brown:</strong> an <strong>control</strong>, octagonal-shaped, with <strong>both left- and right-side</strong> control
connection nodes, such as an <em>action</em> plugin</li>
</ul>
<p>The small yellow circles with numerals show that there are <strong>missing configuration
values</strong> for the different plugins.</p>
<p>Between the icons are grey <strong>connection lines</strong>, with the arrow indicating the direction
of data flow. Solid connection lines indicate data flow; dashed connection lines indicate
control flow.</p>
<p>Now, let’s create a pipeline!</p>
</div>
<div class="section" id="creating-a-simple-pipeline">
<span id="cdap-pipelines-getting-started-simple"></span><h2>Creating a Simple Pipeline<a class="headerlink" href="#creating-a-simple-pipeline" title="Permalink to this headline">🔗</a></h2>
<p>In this example, we’ll create a pipeline that reads log file events from a source,
parses them into separate fields, and writes them as individual records to a table.</p>
<ol class="arabic">
<li><p class="first">Start by clicking on the <em>Stream</em> source in the left panel to add a <em>Stream</em> icon to the canvas.</p>
</li>
<li><p class="first">Click on the disclosure triangle to the left of the <em>Transform</em> label section to show
the <em>Transforms</em>, and then click the <em>LogParser</em> transform to add another icon to the canvas.</p>
</li>
<li><p class="first">Click on the disclosure triangle to the left of the <em>Sink</em> label section to show the
<em>Sinks</em>, and then click the <em>Table</em> transform to add another icon to the canvas.</p>
<p>Your canvas should look like this:</p>
<div class="figure align-center" id="id4" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-1-icons.png"><img alt="../_images/cdap-pipelines-gs-1-1-icons.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-1-icons.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Showing icons</span></p>
</div>
</li>
<li><p class="first">Connect the <em>Stream</em> to the <em>LogParser</em> by clicking on the green connection on the
right-hand side of the <em>Stream</em> and dragging out to the left-side connection of the
<em>LogParser</em> and lifting the mouse-button when you reach it. Your canvas should now look like this:</p>
<div class="figure align-center" id="id5" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-2-connected.png"><img alt="../_images/cdap-pipelines-gs-1-2-connected.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-2-connected.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Showing a connection</span></p>
</div>
</li>
<li><p class="first">Similarly, connect the <em>LogParser</em> to the <em>Table</em> to complete the connections. Your
canvas should now look like this, showing that there are a number of properties to
be completed on each plugin:</p>
<div class="figure align-center" id="id6" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-3-connected.png"><img alt="../_images/cdap-pipelines-gs-1-3-connected.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-3-connected.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Showing connections</span></p>
</div>
<p>(If you make a mistake or need to remove a connection, click and drag on the white
circle just to the right of the connection arrow you’d like to disconnect. When you
drag off that circle and release the mouse, the connection will be deleted and
disappear.)</p>
</li>
<li><p class="first">To set properties of a plugin, click on each icon in turn. When you click an icon, a dialog box
comes up, showing the properties available for each plugin. Any <strong>required</strong> properties
are indicated with a red asterisk after the label. In this case, the <em>Stream</em> icon has
been clicked, and the <em>Label</em>, <em>Stream Name</em>, and <em>Duration</em> are showing as required
properties:</p>
<div class="figure align-center" id="id7" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-4-stream.png"><img alt="../_images/cdap-pipelines-gs-1-4-stream.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-4-stream.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Showing editing of the Stream plugin properties</span></p>
</div>
<p>Enter a stream name, such as <em>demoStream</em>, and a duration, such as <em>1d</em>. As the changes
are automatically saved, you can just close the dialog by either hitting the close button (an <em>X</em> in
the upper-right corner), the <em>escape</em> key on your keyboard, or clicking outside the dialog box.</p>
</li>
<li><p class="first">In a similar fashion, edit the <em>LogParser</em>, setting the <em>Input Name</em> as <em>body</em>, and
accepting the default <em>Log Format</em> of <em>CLF</em>.</p>
<p>Edit the <em>Table</em>, and set the <em>Name</em> as <em>demoTable</em> and the <em>Row Field</em> as <em>ts</em> (for timestamp).</p>
<p>You might notice that the schema displayed has changed as you move from the stream to
the table; the <em>LogParser</em> modifies the schema, breaking the <em>body</em> into the separate
fields we require in the table.</p>
</li>
<li><p class="first">When you are finished, all icons should show that all required fields have been completed
by the absence of yellow circles on the icons.</p>
<p>Name your pipeline by clicking on the text (what else!) <em>Name your pipeline</em> above the
canvas area, and enter a name such as <em>demoPipeline</em>. No spaces are allowed in pipeline
names.</p>
<p>To check that everything is complete, click the <em>Validate</em> button, located in the buttons
above the canvas area. These controls are available in the upper-right:</p>
<div class="figure align-center" id="id8" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-5-buttons.png"><img alt="../_images/cdap-pipelines-gs-1-5-buttons.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-5-buttons.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Button labels, upper-right toolbar</span></p>
</div>
<p>Clicking the <em>Validate</em> button should produce a banner message similar to:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">Validation</span> <span class="n">success</span><span class="o">!</span> <span class="n">Pipeline</span> <span class="n">demoPipeline</span> <span class="n">is</span> <span class="n">valid</span><span class="p">.</span>
</pre></div>
</div>
</li>
<li><p class="first">If there are any errors, correct them before continuing.</p>
<p>Otherwise, click the <em>Publish</em> button: the pipeline configuration will be saved; a CDAP
application will be created, based on the configuration you have set, complete with a
stream and dataset table; and the application will be ready to run.</p>
<p>Note that errors can occur at the publishing phase that were not caught during
validation; resolve those, if any, before continuing.</p>
<p>The view changes to show the completed application:</p>
<div class="figure align-center" id="id9" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-6-pipeline.png"><img alt="../_images/cdap-pipelines-gs-1-6-pipeline.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-6-pipeline.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> Demo pipeline</span></p>
</div>
<p>Though this pipeline view is not editable, clicking the icons will bring up the same dialogs
as before, showing which values have been configured for each stage of the pipeline.</p>
</li>
<li><p class="first">The pipeline view has controls for launching and managing the physical
application; important buttons to note are <em>Run</em> (on the left) and <em>View in
CDAP</em> (on the right):</p>
<div class="figure align-center" id="id10" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-8-pipeline-annotated.png"><img alt="../_images/cdap-pipelines-gs-1-8-pipeline-annotated.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-8-pipeline-annotated.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> Control buttons, pipeline view</span></p>
</div>
</li>
<li><p class="first">Before we can run the pipeline, we need to put data into the stream for the application to
act on. Navigate to the stream by clicking on the <em>View in CDAP</em> button, then the
<em>Datasets</em> button, and then the <em>demoStream</em> button:</p>
<blockquote>
<div><div class="figure align-center" id="id11" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-7-stream.png"><img alt="../_images/cdap-pipelines-gs-1-7-stream.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-7-stream.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP demoStream:</strong> Status page, with <em>Actions</em> menu for sending events</span></p>
</div>
</div></blockquote>
<p>We can send events to the stream by clicking the <em>Actions</em> menu, selecting <em>Send
Events</em>, and then clicking <em>Upload</em> in the dialog that follows. Navigate on your drive
to your CDAP home directory, and locate the file <code class="docutils literal notranslate"><span class="pre">examples/resources/accesslog.txt</span></code>
included in your CDAP Sandbox. This file contains 10,000 records in CLF format.</p>
<p>Once the file has been uploaded successfully by CDAP, you should see the <em>Total Events</em>
for the stream change by 10,000.</p>
<p>Return to <em>CDAP Pipelines</em> by clicking your browser back-button.</p>
</li>
<li><p class="first">You can now run the pipeline. Click the <em>Run</em> button, located in the upper-left. (No
runtime arguments are required; you can click <em>Start Now</em> in the dialog that appears.)</p>
<p>The pipeline should start running, as indicated by the green <em>Running</em> text indicating
the status:</p>
<div class="figure align-center" id="id12" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-9-pipeline-running.png"><img alt="../_images/cdap-pipelines-gs-1-9-pipeline-running.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-9-pipeline-running.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> A <em>Running</em> pipeline</span></p>
</div>
</li>
<li><p class="first">The number of records processed will, in time, change from zero to 10,000.
When the run completes, the status icon will change to <em>Completed</em>. A start time and
duration should appear in the status panel:</p>
<div class="figure align-center" id="id13" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-10-completed.png"><img alt="../_images/cdap-pipelines-gs-1-10-completed.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-10-completed.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> Completed run of <em>demoPipeline</em></span></p>
</div>
</li>
<li><p class="first">You can now check the results by looking at the contents of the <em>demoTable</em>. Rather
than using the <em>View in CDAP</em> button, we’ll use a faster method to find the dataset.
Click on the icon representing the table, to bring up the table configuration. In the
upper-right of the dialog is a <em>Jump</em> button that brings down a menu with two items on
it. The first one takes you directly to the table in CDAP. (The second takes you to the
table in the <a class="reference internal" href="../metadata/metadata-ui.html#metadata-ui"><span class="std std-ref">CDAP Metadata UI</span></a>.)</p>
<div class="figure align-center" id="id14" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-11-jump-button.png"><img alt="../_images/cdap-pipelines-gs-1-11-jump-button.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-11-jump-button.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> <em>Jump</em> button in the pipeline configuration dialog</span></p>
</div>
<p>Navigate to the <em>demoTable</em> dataset, and run a default <em>Explore</em> query that selects the first
five records, by:</p>
<ul class="simple">
<li>clicking <em>Explore</em> and then</li>
<li>clicking <em>Execute SQL</em>:</li>
</ul>
<div class="figure align-center" id="id15" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-12-demotable.png"><img alt="../_images/cdap-pipelines-gs-1-12-demotable.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-12-demotable.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> Results of exploring the <em>demoTable</em></span></p>
</div>
<p>Here you can see that the log records have been successfully loaded into the stream,
parsed by the log parser, and then saved as parsed records to the table. This data is
now available for further analysis, such looking for unique records, sorting, etc.</p>
</li>
</ol>
<p>This completes the <em>Getting Started</em> for CDAP Pipelines.</p>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Introduction to CDAP Pipelines" href="concepts-design.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="CDAP Studio" href="studio.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>